/*     */ package com.zimbra.cs.index.analysis;
/*     */ 
/*     */ import java.io.IOException;
/*     */ import java.io.Reader;
/*     */ import org.apache.lucene.analysis.Tokenizer;
/*     */ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/*     */ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
/*     */ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
/*     */ import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ final class UniversalTokenizer
/*     */   extends Tokenizer
/*     */ {
/*     */   private final UniversalLexer lexer;
/*     */   
/*     */   static enum TokenType
/*     */   {
/*  39 */     ALNUM,  APOSTROPHE,  ACRONYM,  COMPANY,  EMAIL,  HOST,  NUM,  CJK;
/*     */     
/*     */     private TokenType() {} }
/*     */   
/*  43 */   private final CharTermAttribute termAttr = (CharTermAttribute)addAttribute(CharTermAttribute.class);
/*  44 */   private final TypeAttribute typeAttr = (TypeAttribute)addAttribute(TypeAttribute.class);
/*  45 */   private final OffsetAttribute offsetAttr = (OffsetAttribute)addAttribute(OffsetAttribute.class);
/*  46 */   private final PositionIncrementAttribute posIncAttr = (PositionIncrementAttribute)addAttribute(PositionIncrementAttribute.class);
/*  47 */   private int cjk = -1;
/*     */   
/*     */   UniversalTokenizer(Reader in) {
/*  50 */     super(in);
/*  51 */     this.lexer = new UniversalLexer(in);
/*     */   }
/*     */   
/*     */   public boolean incrementToken() throws IOException
/*     */   {
/*  56 */     clearAttributes();
/*     */     
/*  58 */     if (this.cjk >= 0) {
/*  59 */       if (this.cjk + 1 < this.lexer.yylength()) {
/*  60 */         this.lexer.getTerm(this.termAttr, this.cjk, 2);
/*  61 */         setOffset(this.lexer.yychar() + this.cjk, 2);
/*  62 */         this.posIncAttr.setPositionIncrement(1);
/*  63 */         this.typeAttr.setType(TokenType.CJK.name());
/*  64 */         this.cjk += 1;
/*  65 */         return true;
/*     */       }
/*  67 */       this.cjk = -1;
/*     */     }
/*     */     
/*     */ 
/*     */ 
/*  72 */     TokenType type = this.lexer.next();
/*  73 */     if (type == null) {
/*  74 */       return false;
/*     */     }
/*     */     
/*  77 */     if (type == TokenType.CJK) {
/*  78 */       if (this.lexer.yylength() == 1) {
/*  79 */         this.lexer.getTerm(this.termAttr, 0, 1);
/*     */       } else {
/*  81 */         this.lexer.getTerm(this.termAttr, 0, 2);
/*  82 */         this.cjk = 1;
/*     */       }
/*  84 */       setOffset(this.lexer.yychar(), this.termAttr.length());
/*  85 */       this.posIncAttr.setPositionIncrement(1);
/*  86 */       this.typeAttr.setType(type.name());
/*     */     } else {
/*  88 */       this.lexer.getTerm(this.termAttr);
/*  89 */       setOffset(this.lexer.yychar(), this.termAttr.length());
/*  90 */       this.posIncAttr.setPositionIncrement(1);
/*  91 */       this.typeAttr.setType(type.name());
/*     */     }
/*  93 */     return true;
/*     */   }
/*     */   
/*     */ 
/*     */ 
/*     */   public final void end()
/*     */   {
/* 100 */     int offset = this.lexer.yychar() + this.lexer.yylength();
/* 101 */     this.offsetAttr.setOffset(offset, offset);
/*     */   }
/*     */   
/*     */   public void reset() throws IOException
/*     */   {
/* 106 */     super.reset();
/* 107 */     this.lexer.yyreset(this.input);
/*     */   }
/*     */   
/*     */   public void reset(Reader reader) throws IOException
/*     */   {
/* 112 */     super.reset(reader);
/* 113 */     reset();
/*     */   }
/*     */   
/*     */   private void setOffset(int start, int len) {
/* 117 */     this.offsetAttr.setOffset(start, start + len);
/*     */   }
/*     */ }


/* Location:              /home/mint/zimbrastore.jar!/com/zimbra/cs/index/analysis/UniversalTokenizer.class
 * Java compiler version: 7 (51.0)
 * JD-Core Version:       0.7.1
 */